resources.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355
  1. # -*- coding: utf-8 -*-
  2. #
  3. # Copyright (C) 2013-2017 Vinay Sajip.
  4. # Licensed to the Python Software Foundation under a contributor agreement.
  5. # See LICENSE.txt and CONTRIBUTORS.txt.
  6. #
  7. from __future__ import unicode_literals
  8. import bisect
  9. import io
  10. import logging
  11. import os
  12. import pkgutil
  13. import shutil
  14. import sys
  15. import types
  16. import zipimport
  17. from . import DistlibException
  18. from .util import cached_property, get_cache_base, path_to_cache_dir, Cache
  19. logger = logging.getLogger(__name__)
  20. cache = None # created when needed
  21. class ResourceCache(Cache):
  22. def __init__(self, base=None):
  23. if base is None:
  24. # Use native string to avoid issues on 2.x: see Python #20140.
  25. base = os.path.join(get_cache_base(), str('resource-cache'))
  26. super(ResourceCache, self).__init__(base)
  27. def is_stale(self, resource, path):
  28. """
  29. Is the cache stale for the given resource?
  30. :param resource: The :class:`Resource` being cached.
  31. :param path: The path of the resource in the cache.
  32. :return: True if the cache is stale.
  33. """
  34. # Cache invalidation is a hard problem :-)
  35. return True
  36. def get(self, resource):
  37. """
  38. Get a resource into the cache,
  39. :param resource: A :class:`Resource` instance.
  40. :return: The pathname of the resource in the cache.
  41. """
  42. prefix, path = resource.finder.get_cache_info(resource)
  43. if prefix is None:
  44. result = path
  45. else:
  46. result = os.path.join(self.base, self.prefix_to_dir(prefix), path)
  47. dirname = os.path.dirname(result)
  48. if not os.path.isdir(dirname):
  49. os.makedirs(dirname)
  50. if not os.path.exists(result):
  51. stale = True
  52. else:
  53. stale = self.is_stale(resource, path)
  54. if stale:
  55. # write the bytes of the resource to the cache location
  56. with open(result, 'wb') as f:
  57. f.write(resource.bytes)
  58. return result
  59. class ResourceBase(object):
  60. def __init__(self, finder, name):
  61. self.finder = finder
  62. self.name = name
  63. class Resource(ResourceBase):
  64. """
  65. A class representing an in-package resource, such as a data file. This is
  66. not normally instantiated by user code, but rather by a
  67. :class:`ResourceFinder` which manages the resource.
  68. """
  69. is_container = False # Backwards compatibility
  70. def as_stream(self):
  71. """
  72. Get the resource as a stream.
  73. This is not a property to make it obvious that it returns a new stream
  74. each time.
  75. """
  76. return self.finder.get_stream(self)
  77. @cached_property
  78. def file_path(self):
  79. global cache
  80. if cache is None:
  81. cache = ResourceCache()
  82. return cache.get(self)
  83. @cached_property
  84. def bytes(self):
  85. return self.finder.get_bytes(self)
  86. @cached_property
  87. def size(self):
  88. return self.finder.get_size(self)
  89. class ResourceContainer(ResourceBase):
  90. is_container = True # Backwards compatibility
  91. @cached_property
  92. def resources(self):
  93. return self.finder.get_resources(self)
  94. class ResourceFinder(object):
  95. """
  96. Resource finder for file system resources.
  97. """
  98. if sys.platform.startswith('java'):
  99. skipped_extensions = ('.pyc', '.pyo', '.class')
  100. else:
  101. skipped_extensions = ('.pyc', '.pyo')
  102. def __init__(self, module):
  103. self.module = module
  104. self.loader = getattr(module, '__loader__', None)
  105. self.base = os.path.dirname(getattr(module, '__file__', ''))
  106. def _adjust_path(self, path):
  107. return os.path.realpath(path)
  108. def _make_path(self, resource_name):
  109. # Issue #50: need to preserve type of path on Python 2.x
  110. # like os.path._get_sep
  111. if isinstance(resource_name, bytes): # should only happen on 2.x
  112. sep = b'/'
  113. else:
  114. sep = '/'
  115. parts = resource_name.split(sep)
  116. parts.insert(0, self.base)
  117. result = os.path.join(*parts)
  118. return self._adjust_path(result)
  119. def _find(self, path):
  120. return os.path.exists(path)
  121. def get_cache_info(self, resource):
  122. return None, resource.path
  123. def find(self, resource_name):
  124. path = self._make_path(resource_name)
  125. if not self._find(path):
  126. result = None
  127. else:
  128. if self._is_directory(path):
  129. result = ResourceContainer(self, resource_name)
  130. else:
  131. result = Resource(self, resource_name)
  132. result.path = path
  133. return result
  134. def get_stream(self, resource):
  135. return open(resource.path, 'rb')
  136. def get_bytes(self, resource):
  137. with open(resource.path, 'rb') as f:
  138. return f.read()
  139. def get_size(self, resource):
  140. return os.path.getsize(resource.path)
  141. def get_resources(self, resource):
  142. def allowed(f):
  143. return (f != '__pycache__' and not
  144. f.endswith(self.skipped_extensions))
  145. return set([f for f in os.listdir(resource.path) if allowed(f)])
  146. def is_container(self, resource):
  147. return self._is_directory(resource.path)
  148. _is_directory = staticmethod(os.path.isdir)
  149. def iterator(self, resource_name):
  150. resource = self.find(resource_name)
  151. if resource is not None:
  152. todo = [resource]
  153. while todo:
  154. resource = todo.pop(0)
  155. yield resource
  156. if resource.is_container:
  157. rname = resource.name
  158. for name in resource.resources:
  159. if not rname:
  160. new_name = name
  161. else:
  162. new_name = '/'.join([rname, name])
  163. child = self.find(new_name)
  164. if child.is_container:
  165. todo.append(child)
  166. else:
  167. yield child
  168. class ZipResourceFinder(ResourceFinder):
  169. """
  170. Resource finder for resources in .zip files.
  171. """
  172. def __init__(self, module):
  173. super(ZipResourceFinder, self).__init__(module)
  174. archive = self.loader.archive
  175. self.prefix_len = 1 + len(archive)
  176. # PyPy doesn't have a _files attr on zipimporter, and you can't set one
  177. if hasattr(self.loader, '_files'):
  178. self._files = self.loader._files
  179. else:
  180. self._files = zipimport._zip_directory_cache[archive]
  181. self.index = sorted(self._files)
  182. def _adjust_path(self, path):
  183. return path
  184. def _find(self, path):
  185. path = path[self.prefix_len:]
  186. if path in self._files:
  187. result = True
  188. else:
  189. if path and path[-1] != os.sep:
  190. path = path + os.sep
  191. i = bisect.bisect(self.index, path)
  192. try:
  193. result = self.index[i].startswith(path)
  194. except IndexError:
  195. result = False
  196. if not result:
  197. logger.debug('_find failed: %r %r', path, self.loader.prefix)
  198. else:
  199. logger.debug('_find worked: %r %r', path, self.loader.prefix)
  200. return result
  201. def get_cache_info(self, resource):
  202. prefix = self.loader.archive
  203. path = resource.path[1 + len(prefix):]
  204. return prefix, path
  205. def get_bytes(self, resource):
  206. return self.loader.get_data(resource.path)
  207. def get_stream(self, resource):
  208. return io.BytesIO(self.get_bytes(resource))
  209. def get_size(self, resource):
  210. path = resource.path[self.prefix_len:]
  211. return self._files[path][3]
  212. def get_resources(self, resource):
  213. path = resource.path[self.prefix_len:]
  214. if path and path[-1] != os.sep:
  215. path += os.sep
  216. plen = len(path)
  217. result = set()
  218. i = bisect.bisect(self.index, path)
  219. while i < len(self.index):
  220. if not self.index[i].startswith(path):
  221. break
  222. s = self.index[i][plen:]
  223. result.add(s.split(os.sep, 1)[0]) # only immediate children
  224. i += 1
  225. return result
  226. def _is_directory(self, path):
  227. path = path[self.prefix_len:]
  228. if path and path[-1] != os.sep:
  229. path += os.sep
  230. i = bisect.bisect(self.index, path)
  231. try:
  232. result = self.index[i].startswith(path)
  233. except IndexError:
  234. result = False
  235. return result
  236. _finder_registry = {
  237. type(None): ResourceFinder,
  238. zipimport.zipimporter: ZipResourceFinder
  239. }
  240. try:
  241. # In Python 3.6, _frozen_importlib -> _frozen_importlib_external
  242. try:
  243. import _frozen_importlib_external as _fi
  244. except ImportError:
  245. import _frozen_importlib as _fi
  246. _finder_registry[_fi.SourceFileLoader] = ResourceFinder
  247. _finder_registry[_fi.FileFinder] = ResourceFinder
  248. del _fi
  249. except (ImportError, AttributeError):
  250. pass
  251. def register_finder(loader, finder_maker):
  252. _finder_registry[type(loader)] = finder_maker
  253. _finder_cache = {}
  254. def finder(package):
  255. """
  256. Return a resource finder for a package.
  257. :param package: The name of the package.
  258. :return: A :class:`ResourceFinder` instance for the package.
  259. """
  260. if package in _finder_cache:
  261. result = _finder_cache[package]
  262. else:
  263. if package not in sys.modules:
  264. __import__(package)
  265. module = sys.modules[package]
  266. path = getattr(module, '__path__', None)
  267. if path is None:
  268. raise DistlibException('You cannot get a finder for a module, '
  269. 'only for a package')
  270. loader = getattr(module, '__loader__', None)
  271. finder_maker = _finder_registry.get(type(loader))
  272. if finder_maker is None:
  273. raise DistlibException('Unable to locate finder for %r' % package)
  274. result = finder_maker(module)
  275. _finder_cache[package] = result
  276. return result
  277. _dummy_module = types.ModuleType(str('__dummy__'))
  278. def finder_for_path(path):
  279. """
  280. Return a resource finder for a path, which should represent a container.
  281. :param path: The path.
  282. :return: A :class:`ResourceFinder` instance for the path.
  283. """
  284. result = None
  285. # calls any path hooks, gets importer into cache
  286. pkgutil.get_importer(path)
  287. loader = sys.path_importer_cache.get(path)
  288. finder = _finder_registry.get(type(loader))
  289. if finder:
  290. module = _dummy_module
  291. module.__file__ = os.path.join(path, '')
  292. module.__loader__ = loader
  293. result = finder(module)
  294. return result